package com.shujia.spark.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo12Union {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("union")

    val sc = new SparkContext(conf)

    /**
     * 基于一个集合构建RDD
     *
     */

    val RDD1: RDD[Int] = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7))


    val RDD2: RDD[Int] = sc.parallelize(List(5, 6, 7, 8, 9, 10))


    /**
     * union: 合并两个RDD,rdd的类型必须一致
     * union的合并再数据层面美欧合并，只是再逻辑层面合并了
     */
    val unionRDD: RDD[Int] = RDD1.union(RDD2)

    unionRDD.foreach(println)

    /**
     * distinct: 去重重复数据,会产生shuffle
     *
     */
    val distinctRDD: RDD[Int] = unionRDD.distinct()

    distinctRDD.foreach(println)

  }

}
